package com.chance.cc.crawler.development.scripts.douyin.api;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.*;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.google.common.collect.Maps;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.security.NoSuchAlgorithmException;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;

/**
 * @author bx
 * @date 2020/12/6 0006 12:52
 */
public class DYCommentCrawlerScript extends CrawlerCommonScript {

    private Downloader httpDownload;
    private static Object httpDownloadObj = new Object();
    private static HttpConfig dySignatureConfig = HttpConfig.me("signature");

    private static Logger logger = LoggerFactory.getLogger(DYCommentCrawlerScript.class);
    public static final String site = "api-comment";

    public static final String commentUrlRegular = "https://\\S*/aweme/v2/comment/list/\\S*";
    public static final String commentUrl = "https://api5-normal-c-lf.amemv.com/aweme/v2/comment/list/?";
    public static final String commentStartUrl = "https://start/aweme/v2/comment/list/?";
//    public static final String commentUrlParamFormat = "aweme_id=%s" +
//            "&cursor=%s" +
//            "&count=20&address_book_access=2&gps_access=2&forward_page_type=1&channel_id=0&city=310000&hotsoon_filtered_count=0&hotsoon_has_more=0&follower_count=0&is_familiar=0&page_source=0&user_avatar_shrink=96_96" +
//            "&aweme_author=%s" +
//            "&manifest_version_code=140101" +
//            "&_rticket=%s" +
//            "&app_type=normal&iid=%s&channel=%s&device_type=%s" +
//            "&language=zh&cpu_support64=true&host_abi=%s&resolution=%s" +
//            "&openudid=%s&update_version_code=14109900&cdid=%s" +
//            "&appTheme=dark&os_api=%s&dpi=280&ac=wifi&device_id=%s&mcc_mnc=%s&os_version=%s&version_code=140100" +
//            "&app_name=aweme&version_name=14.1.0&device_brand=%s&ssmix=a&device_platform=android&aid=1128&ts=%s";
    public static final String commentUrlParamFormat = "aweme_id=%s" +
        "&cursor=%s" +
        "&count=50" +
        "&aweme_author=%s" +
        "&iid=%s&channel=%s&device_type=%s" +
        "&device_id=%s&os_version=%s&version_code=140100" +
        "&app_name=aweme&device_platform=android&aid=1128";

    public static final String useragentFormat = "com.ss.android.ugc.aweme/140101 (Linux; U; Android 8.0.0; zh_CN; MI 5s;" +
            " Build/OPR1.170623.032; Cronet/TTNetVersion:%s 2020-12-16 QuicVersion:%s 2020-10-14)";

    private LinkedBlockingQueue<String> devicesQueue = new LinkedBlockingQueue<>();

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    private static final String hookApi = "http://192.168.1.212:8383";
    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        String useragent = crawlerRecord.getHttpRequest().getHeaders().get("User-Agent");
        List<CrawlerRequestRecord> crawlerRequestRecords = new ArrayList<>();
        if (!page.isDownloadSuccess()){
            logger.error("dy video comment request download has error status code {},page raw text [{}],will retry!",
                    page.getStatusCode(),page.getRawText());
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
            return crawlerRequestRecords;
        }

        try{
            if(page.getRawText().contains("暂时没有更多了")
                    ||page.getRawText().contains("评论因未授权暂不支持查看")
                    ||(page.getRawText().contains("\"comments\":null") && !page.getRawText().contains("\"total\":")) //视频没了
            ){
                crawlerRecord.setNeedWashPage(false);
                return crawlerRequestRecords;
            }
            Json accountResultJson = new Json(page.getRawText());
            String statusCode = accountResultJson.jsonPath($_type + ".status_code").get();
            String hasMore = accountResultJson.jsonPath($_type + ".has_more").get();
            Integer total = Integer.valueOf(accountResultJson.jsonPath($_type + ".total").get());

            if (total < 1){
                throw new Exception("dy account next page total error:"+total);
            }

            List<String> comments = accountResultJson.jsonPath($_type + ".comments").all();
            List<String> images = new ArrayList<>();
//            if (comments != null && comments.size() > 0){
//                for (String comment : comments) {
//                    try {
//                        Json commentJson = new Json(comment);
//                        List<String> url_list = commentJson.jsonPath($_type + ".user.avatar_medium.url_list").all();
//                        if (url_list != null){
//                            for (String url : url_list) {
//                                if (url.contains("p3.douyinpic.com")){
//                                    images.add(url);
//                                }
//                            }
//                        }
//                    } catch (Exception e) {
//                    }
//                }
//            }

            if (images.size() > 0 ){
                for (String image : images) {
                    internalDownloadImg(crawlerRequestRecords,crawlerRecord,image,useragent);
                }
            }

            if ("0".equals(statusCode) && "1".equals(hasMore)){
                String cursor = accountResultJson.jsonPath($_type + ".cursor").get();
                if ("0".equals(cursor)){ //有下一页，但是游标出错，重试
                    throw new Exception("dy account next page cursor error:"+cursor);
                }
                if (crawlerRecord.tagsCreator().bizTags().hasKVTag("download_retry_count")){
                    crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKvTags().remove("download_retry_count");
                }
                CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRecord)
                        .httpUrl(commentUrl+UUID.randomUUID().toString())
                        .httpHeads(page.getRequest().getHeaders())
                        .releaseTime(System.currentTimeMillis())
                        .needWashed(true)
                        .copyBizTags()
                        .build();
                logger.info("current crawler record turn page filter info:{}",crawlerRecord.getTurnPageFilterInfo());
                logger.info("next crawler record turn page filter info:{}",crawlerRequestRecord.getTurnPageFilterInfo());
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("download_retry_count",0);
                crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("cursor",cursor);
                crawlerRequestRecords.add(crawlerRequestRecord);
            }
        }catch (Exception e){
            logger.error("dy video comment request url {} download error, page raw text {}"
                    ,page.getRequest().getUrl(),page.getRawText());
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
        }

        return crawlerRequestRecords;
    }

    private void internalDownloadImg(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord,String url,String useragent) throws NoSuchAlgorithmException {
        if (StringUtils.isNotBlank(url)){
            CrawlerRequestRecord imgRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .httpHead("Host","p3.douyinpic.com")
                    .httpHead("User-Agent",useragent)
                    .releaseTime(System.currentTimeMillis())
                    .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                    .build();
            imgRecord.getHttpConfig().setResponseTextGenerateHtml(false);
            crawlerRequestRecords.add(imgRecord);
        }
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();

        Json searchJson = new Json(page.getRawText());
        List<String> comments = searchJson.jsonPath($_type + ".comments").all();
        if (comments != null){
            for(String comment : comments){
                try {
                    Json dataJson = new Json(comment);
                    String cid = dataJson.jsonPath($_type + ".cid").get();
                    String aweme_id = dataJson.jsonPath($_type + ".aweme_id").get();
                    String create_time = dataJson.jsonPath($_type + ".create_time").get();

                    CrawlerData crawlerData = CrawlerData.builder()
                            .data(crawlerRecord, page)
                            .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.comment.enumVal(), cid))
                            .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), aweme_id))
                            .url("")
                            .releaseTime(Long.valueOf(create_time ) * 1000L)
                            .content(comment)
                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                            .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                            .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                            .isEndTimeFromSchedulerTime(false)
                            .build();

                    crawlerDataList.add(crawlerData);
                } catch (Exception e) {
                    logger.error("dy video comment content data normal,error: {}",e.getMessage());
                }
            }
        }
        return crawlerDataList;
    }

    @Override
    public String domain() {
        return "api-dy";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(commentStartUrl); //start url--只是为了进入脚本没有实际意义
        addUrlRegular(commentUrlRegular);

    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {

        String recordSite = crawlerRecord.tagsCreator().bizTags().site();
        if (site.equals(recordSite)){
            return true;
        }
        return false;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        requestRecord.setNeedWashPage(true);
        List<CrawlerRecord> allItemRecords = null;
        if (commentStartUrl.equals(requestRecord.getHttpRequest().getUrl())){
            if (supportSourceRecords == null || supportSourceRecords.size() < 1){
                throw new RuntimeException("support source records must be configured!");
            }
        }

        if (supportSourceRecords != null && supportSourceRecords.size() >1){
            for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
                if (supportSourceRecord.getHttpRequest().getUrl().contains("/crawler/oauth/api/v1/douyin/userOauthInfos")){
                    devicesQueue.clear();
                    initAuthorInfos(supportSourceRecord); //初始 认证信息
                }
            }
            if (devicesQueue.size() < 1){
                throw new RuntimeException("Must initial device list!");
            }
        } else {
            waitDeviceInit();

            processCrawlerRecord(requestRecord);
        }


//        requestRecord.getHttpRequest().addHeader("Cookie", "odin_tt=3c383384b62f3a522b42bf27c135f9194c3b442c6a3323cb36487a201062da78e91cb4155487dd26a178295f112e0f24d899d14689c11b320e2d28fa856fb9f5");
        return super.prepareRequest(requestRecord, supportSourceRecords);
    }

    private void processCrawlerRecord(CrawlerRequestRecord requestRecord){
        String aweme_id = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("aweme_id");
        String cursor = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("cursor");
        String aweme_author = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("aweme_author");
        long rticket = System.currentTimeMillis();
        long ts = rticket/1000;

        String deivce = pollOneDevice();
        Map map = JSON.parseObject(deivce, Map.class);
        String iid = map.get("iid").toString();
        String device_model = map.get("device_model").toString();
        String cpu_abi = map.get("cpu_abi").toString();
        String resolution = map.get("resolution").toString();
        String openudid = map.get("openudid").toString();
        String cdid = map.get("cdid").toString();
        String os_api = map.get("os_api").toString();
        String device_id = map.get("device_id").toString();
        String mcc_mnc = map.get("mcc_mnc").toString();
        String os_version = map.get("os_version").toString();
        String device_brand = map.get("device_manufacturer").toString();
        Object channelObject = map.get("channel");
        String channel =  channelObject==null ? "huawei_1" : channelObject.toString();
//        String channle = RandomStringUtils.random(8,"abcdefghijklmnopqqrsduvwxyz1234567890_");
//        String channle = "huawei_1";
//        String channle = "huawei_"+ RandomUtils.nextInt(1,100);

//        String commentUrlParam = String.format(commentUrlParamFormat,aweme_id,cursor,aweme_author,rticket,
//                iid,channle,device_model,cpu_abi, resolution,openudid,cdid,os_api,device_id,mcc_mnc,os_version,device_brand,ts);
        String commentUrlParam = String.format(commentUrlParamFormat,aweme_id,cursor,aweme_author,iid,channel,
                device_model,device_id,os_version);
        requestRecord.getHttpRequest().setUrl(commentUrl+commentUrlParam);

        Map<String, String> signatureMap = dySignature( requestRecord.getHttpRequest().getUrl(), Maps.newHashMap());
        requestRecord.getHttpConfig().setProxy(proxy);
        requestRecord.getHttpConfig().setUseProxy(true);
        requestRecord.getHttpConfig().setHttpSite("comment_new");
        requestRecord.getHttpConfig().setDisableCookie(true);
        requestRecord.getHttpRequest().getHeaders().putAll(signatureMap);
        String useragent = String.format(useragentFormat, RandomStringUtils.random(8,"0123456789abcdf"),RandomStringUtils.random(8,"0123456789abcdf"));
        requestRecord.getHttpRequest().getHeaders().put("User-Agent",useragent);
        requestRecord.getHttpRequest().getHeaders().put("Host","api5-normal-c-lf.amemv.com");
        devicesQueue.add(deivce);//返还设备
    }

    private void waitDeviceInit(){
        while (devicesQueue.size() <50){//初始设备容量
            try {
                Thread.sleep(1000);
                logger.info("dy comment wait for the device to initialize...");
            } catch (InterruptedException e) {
                logger.error(e.getMessage());
            }
        }
    }

    private String pollOneDevice(){
        String device = "";
        try {

            device = devicesQueue.poll(3, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            logger.error(e.getMessage());
        }
        return device;
    }

    private Map<String,String> dySignature(String dy_url,Map<String,String> dataMap){
        String signatureUrl = hookApi + "/frida/dy/signature";

        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(signatureUrl);
        Map<String,Object> dyDataMap = new HashMap<>();
        dyDataMap.put("dy_url",dy_url);
        dyDataMap.put("map_data_json",JSON.toJSONString(dataMap));
        httpRequest.setRequestBody(HttpRequestBody.form(dyDataMap,"utf-8"));
        httpRequest.setMethod("POST");
        String rawText = null;
        HttpPage download = null;
        try {
            download = httpDownload.download(httpRequest, dySignatureConfig);
            rawText = download.getRawText();
            if(rawText.contains("DOCTYPE HTML PUBLIC")){
                throw new Exception("DOCTYPE HTML PUBLIC");
            }
        } catch (Exception e) {
            try {
                while (download == null || !download.isDownloadSuccess() || StringUtils.isBlank(rawText)){
                    Thread.sleep(60000);
                    download = httpDownload.download(httpRequest, dySignatureConfig);
                    rawText = download.getRawText();
                }
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
        return JSON.parseObject(rawText,Map.class);
    }

    @Override
    public void beforeDownload(CrawlerRecordContext context) {
        if (httpDownload == null){
            synchronized (httpDownloadObj) {
                if (httpDownload == null){
                    httpDownload = context.getPageDownloader();
                }
            }
        }
        super.beforeDownload(context);
    }

    private void addCrawlerRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord){

        int count = 1;
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        if (crawlerBusinessTags.hasKVTag("download_retry_count")){
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag("download_retry_count").getVal();
            if (count >= 20){
                logger.error("dy video comment download he number of retries exceeds the limit" +
                        ",request url {},download detail {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        count++;
        crawlerBusinessTags.addCustomKV("download_retry_count",count);

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpRequest(crawlerRecord.getHttpRequest())
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .notFilterRecord()
                .needWashed(true)
                .build();
        crawlerRequestRecord.setTurnPageFilterInfo(crawlerRecord.getTurnPageFilterInfo());
        crawlerRequestRecords.add(crawlerRequestRecord);

    }

    private void initAuthorInfos(CrawlerRequestRecord supportSourceRecord){
        List<String> devices = new ArrayList<>(); //可用设备列表
        try {
            HttpPage httpPage = supportSourceRecord.getInternalDownloadPage();
            Json rawText = new Json(httpPage.getRawText());
            String status = rawText.jsonPath($_type + ".status").get();
            List<String> contents = rawText.jsonPath($_type + ".content").all();

            if ("0".equals(status) && contents != null && contents.size() > 0){

                List<Map> userOauthInfos = new Json(contents.get(0)).toList(Map.class);
                for (Map userOauthInfo : userOauthInfos) {
                    String oauthInfo = String.valueOf(userOauthInfo.get("oauthInfo"));
                    Json oauthInfoJson = new Json(oauthInfo);
                    String category = oauthInfoJson.jsonPath($_type + ".category").get();

                    if ("device_comment".equals(category)){
                        devices.add(oauthInfo);
                    }
                }
            }
            if (devices.size() > 0){
                Collections.reverse(devices);
                devicesQueue.addAll(devices);
            }
            logger.info("comment devices count {}",devicesQueue.size());
        } catch (Exception e) {
            logger.error(e.getMessage(),e);
        }
    }
}
